#include "CleanPage.h"
#include "Mylogger.h"
#include "tinyxml2.h"
#include <iostream>
#include <regex>

using std::cerr;
using std::endl;
using std::ifstream;
using namespace tinyxml2;

bool isUTF8(const char* str) {
  unsigned int nBytes = 0;//UFT8可用1-6个字节编码,ASCII用一个字节
  unsigned char chr = *str;
  bool bAllAscii = true;
  for (unsigned int i = 0; str[i] != '\0'; ++i){
    chr = *(str + i);
    //判断是否ASCII编码,如果不是,说明有可能是UTF8,ASCII用7位编码,最高位标记为0,0xxxxxxx
    if (nBytes == 0 && (chr & 0x80) != 0){
      bAllAscii = false;
    }
    if (nBytes == 0) {
      //如果不是ASCII码,应该是多字节符,计算字节数
      if (chr >= 0x80) {
        if (chr >= 0xFC && chr <= 0xFD){
          nBytes = 6;
        }
        else if (chr >= 0xF8){
          nBytes = 5;
        }
        else if (chr >= 0xF0){
          nBytes = 4;
        }
        else if (chr >= 0xE0){
          nBytes = 3;
        }
        else if (chr >= 0xC0){
          nBytes = 2;
        }
        else{
          return false;
        }
        nBytes--;
      }
    }
    else{
      //多字节符的非首字节,应为 10xxxxxx
      if ((chr & 0xC0) != 0x80){
        return false;
      }
      //减到为零为止
      nBytes--;
    }
  }
  //违返UTF8编码规则
  if (nBytes != 0) {
    return false;
  }
  if (bAllAscii){ //如果全部都是ASCII, 也是UTF8
    return true;
  }
  return true;
}

void CleanPage::cleanSingleWebPage(const string& filename){
    //1.清空vector
    _cleanedWebPage.clear();
    //2.载入文件
    XMLDocument doc;
    doc.LoadFile(filename.c_str());

    //3.解析文件
    if(doc.ErrorID()){
        cerr <<filename<<"LoadFile error" << endl;
        return ;
    }

    XMLElement * itemNode = doc.FirstChildElement("rss")->FirstChildElement("channel")
        ->FirstChildElement("item");

    while(itemNode){
        string title  = itemNode->FirstChildElement("title")->GetText();
        /* std::cout << "title = " << title << "\n"; */
        //判断是否是UTF_8
        int isutf_8 = true;
        isutf_8 = isUTF8(title.c_str());
        if(isutf_8 == false){
            cout <<filename <<": title is not UTF_8\n";
            break;
        }
        string link  = itemNode->FirstChildElement("link")->GetText();
        //判断是否是UTF_8
        isutf_8 = isUTF8(link.c_str());
        if(isutf_8 == false){
            cout <<filename <<": link is not UTF_8\n";
            break;
        }
        std::regex htmlTagPattern("</?[^>]+>");
        string newDescription;
        string newContent;
        auto descElement = itemNode->FirstChildElement("description");
        if(descElement!=nullptr){
            string description  = itemNode->FirstChildElement("description")->GetText();
            newDescription = std::regex_replace(description,htmlTagPattern,"");
        }
        auto conElement = itemNode->FirstChildElement("content");
        if(conElement!=nullptr){
            string content  = conElement->GetText();
            string newContent = std::regex_replace(content,htmlTagPattern,"");
        }

        //4.计算指纹,并将其存入vector中
        string contentText = newDescription + newContent;
        //判断是否是UTF_8编码,如果不是,就跳过此篇文章的解析
        isutf_8 = isUTF8(contentText.c_str());
        if(isutf_8 == false){
            cout <<filename <<": content is not UTF_8\n";
            break;
        }
        size_t topN = 5;
        uint64_t u64 = 0;
        _simhasher.make(contentText,topN,u64);

        if(u64 != 0){   //如果有内容
            _fingerprint.push_back(make_pair(_fingerprint.size()+1,u64));
            /* cout << "_fingerprint.size = " << _fingerprint.size() << endl; */
            //5.将解析后的结果加入vector
            WebPage webPage;
            webPage._docId = 0; //暂时是一个无效docId
            webPage._docTitle = title;
            webPage._docUrl = link;
            webPage._docContent = contentText; 
            _cleanedWebPage.push_back(webPage);
        }

        //遍历
        itemNode = itemNode->NextSiblingElement("item");
    }
}

void CleanPage::storeSinglePaper(const WebPage& webPage,ofstream& ofs){
    //将文章的内容写入磁盘
    //1.构建xml格式的文章
    //创建一个XML的文档对象
    XMLDocument* myDocument = new XMLDocument();
    //创建一个doc元素节点
    XMLElement *doc = myDocument->NewElement("doc");
    myDocument->LinkEndChild(doc);

    //创建一个docid元素并连接
    XMLElement * docid = myDocument->NewElement("docid"); 
    doc->LinkEndChild(docid);
    XMLText * docidText = myDocument->NewText(std::to_string(webPage._docId).c_str());
    docid->LinkEndChild(docidText);

    //创建一个title元素并连接
    XMLElement * title = myDocument->NewElement("title"); 
    doc->LinkEndChild(title);
    XMLText * titleText = myDocument->NewText(webPage._docTitle.c_str());
    title->LinkEndChild(titleText);

    //创建一个link元素并连接
    XMLElement * link = myDocument->NewElement("link"); 
    doc->LinkEndChild(link);
    XMLText * linkText = myDocument->NewText(webPage._docUrl.c_str());
    link->LinkEndChild(linkText);

    //创建一个content元素并连接
    XMLElement * content = myDocument->NewElement("content"); 
    doc->LinkEndChild(content);
    XMLText * contentText = myDocument->NewText(webPage._docContent.c_str());
    content->LinkEndChild(contentText);

    //2.将xml格式内容转化为字符串
    tinyxml2::XMLPrinter printer;
    myDocument->Print(&printer);
    std::stringstream ss;
    ss << printer.CStr();
    std::string xmlString = ss.str();
    delete myDocument;
    //3.将字符串写入磁盘
    ofs << xmlString;
}

void CleanPage::clean(){
    //读取配置文件
    LogInfo("Configuration file reading...");
    _conf.buildConf();
    LogInfo("The configuration file has been read");
    //清空_cleanedIndex
    _cleanedIndex.clear();
    //1.打开文件
    ofstream ofs(_cleanedFilename.c_str());
    int docid = 1;
    std::streampos oldPos = 0;
    std::streampos newPos = 0;
    //2.遍历conf中的_webPageFiles,得到每个webPage的文件名
    LogInfo("File cleaning...");
    for(const string& filename : _conf._webPageFiles){
        //3.清洗文件
        size_t pos = filename.find_last_of("/");
        string fname = filename.substr(pos+1);
        string info = "file cleaning:" + fname;
        LogInfo(info);
        cleanSingleWebPage(filename);
        for(WebPage& webPage : _cleanedWebPage){
            oldPos = ofs.tellp();
            ofs.seekp(oldPos);
            //4.将清洗后的文件存入磁盘
            webPage._docId = docid;
            storeSinglePaper(webPage,ofs);
            newPos = ofs.tellp();
            int length = newPos - oldPos;
            /* if(webPage._docId == 2838){ */
                /* cout << "length = " << length << endl; */
                /* cout << "oldPos = " << oldPos << endl; */
                /* cout << "newPos = " << newPos << endl; */
            /* } */
            //5.修改清洗后的网页偏移库
            _cleanedIndex[webPage._docId] = make_pair(oldPos,length);

            /* cout << "docid = " << docid << endl; */
            ++docid;
        }
    }
    LogInfo("File cleaning finish");
    //6.关闭文件
    ofs.close();
    
    /* //遍历一遍_cleanedIndex */
    /* for(auto begin = _cleanedIndex.begin();begin!=_cleanedIndex.end();++begin){ */
    /*     cout << begin->first << " " << begin->second.first << " " */
    /*         << begin->second.second << endl; */
    /* } */
}

void CleanPage::reDuplicate(){
    vector<pair<int,uint64_t>> temp; 
    temp.reserve(100);
    //将重复的指纹的docid设为0;
    for(int idx=0;idx < (int)_fingerprint.size();++idx){
        /* cout << "size() =" << _fingerprint.size() <<"idx = " << idx << "\n"; */
        if(_fingerprint[idx].first!=0){ //已经被去重的就不在遍历
            uint64_t u1 = _fingerprint[idx].second;
            for(int jdx=idx+1;jdx < (int)_fingerprint.size();++jdx){
                if(_fingerprint[jdx].first == 0){   //跳过已经被去重的部分
                    break;
                }
                uint64_t u2 = _fingerprint[jdx].second;
                if(Simhasher::isEqual(u1,u2)){
                    //删除索引中重复的部分
                    /* cout << "_fingerprint[jdx].first = " << _fingerprint[jdx].first << endl; */
                    auto it = _cleanedIndex.find(_fingerprint[jdx].first);
                    /* cout << "docid = " << it->first << endl; */
                    /* cout << "size() =  " << _cleanedIndex.size() << endl; */
                    /* if(it!=_cleanedIndex.end()){ */
                        _cleanedIndex.erase(it);
                    /* } */
                    _fingerprint[jdx].first = 0;
                }
            }
        }
    }
    //将去重后的内容存入temp中
    for(int idx=0;idx < (int)_fingerprint.size();++idx){
        if(_fingerprint[idx].first != 0){
            temp.push_back(_fingerprint[idx]);
        }
    }
    //交换temp和_fingerprint中的内容
    temp.swap(_fingerprint);

}

void CleanPage::storeIndex(){
    //1.打开文件
    ofstream ofs(_noDupIndexFilename);
    //2,将_clearnIndex的内容写入磁盘
    for(const pair<int,pair<int,int>>& elem:_cleanedIndex){
        ofs << elem.first << " ";
        ofs << elem.second.first << " ";
        ofs << elem.second.second << " ";
        ofs << "\n";
    }
    //3.关闭文件
    ofs.close();
}

void CleanPage::start(){
    //1.清洗文件
    clean();
    //2.文章去重
    LogInfo("Article de-duplication...");
    reDuplicate();
    LogInfo("Article de-duplication has been completed");
    //3.将去重后的索引存入磁盘
    storeIndex();
}
